{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# You may need to restart your runtime prior to this, to let your installation take effect\n",
    "# Some basic setup:\n",
    "# Setup detectron2 logger\n",
    "import detectron2\n",
    "from detectron2.utils.logger import setup_logger\n",
    "setup_logger()\n",
    "\n",
    "# import some common libraries\n",
    "import numpy as np\n",
    "import cv2\n",
    "import random\n",
    "# from google.colab.patches import cv2_imshow\n",
    "\n",
    "# import some common detectron2 utilities\n",
    "from detectron2 import model_zoo\n",
    "from detectron2.engine import DefaultPredictor\n",
    "from detectron2.config import get_cfg\n",
    "from detectron2.utils.visualizer import Visualizer\n",
    "from detectron2.data import MetadataCatalog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.pylab as pylab\n",
    "pylab.rcParams['figure.figsize'] = 13,13\n",
    "def imshow(img):\n",
    "    plt.imshow(img[:, :, [2, 1, 0]])\n",
    "    plt.axis(\"off\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run a pre-trained detectron2 model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget http://images.cocodataset.org/val2017/000000439715.jpg -O input.jpg\n",
    "im = cv2.imread(\"./input.jpg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imshow(im)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cfg = get_cfg()\n",
    "# add project-specific config (e.g., TensorMask) here if you're not running a model in detectron2's core library\n",
    "cfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\n",
    "cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model\n",
    "# Find a model from detectron2's model zoo. You can use the https://dl.fbaipublicfiles... url as well\n",
    "cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")\n",
    "cfg.DATASETS.TRAIN = (\"amodal_train\",)\n",
    "cfg.DATASETS.TEST = (\"small_amodal_test\")\n",
    "predictor = DefaultPredictor(cfg)\n",
    "outputs = predictor(im)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# look at the outputs. See https://detectron2.readthedocs.io/tutorials/models.html#model-output-format for specification\n",
    "outputs[\"instances\"].pred_classes\n",
    "outputs[\"instances\"].pred_boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can use `Visualizer` to draw the predictions on the image.\n",
    "v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\n",
    "v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imshow(v.get_image()[:, :, ::-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from detectron2.engine import DefaultTrainer\n",
    "trainer = DefaultTrainer(cfg) \n",
    "trainer.resume_or_load(resume=False)\n",
    "#  evaluate its performance using AP metric implemented in COCO API.\n",
    "from detectron2.evaluation import COCOEvaluator, inference_on_dataset\n",
    "from detectron2.data import build_detection_test_loader\n",
    "evaluator = COCOEvaluator(\"amodal_val\", cfg, False, output_dir=\"./output/\")\n",
    "val_loader = build_detection_test_loader(cfg, \"amodal_val\")\n",
    "inference_on_dataset(trainer.model, val_loader, evaluator)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train pre-trained model on a custom dataset (FT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[02/29 01:22:13 d2.data.datasets.coco]: \u001b[0mLoaded 2276 images in COCO format from datasets/coco/annotations/COCO_amodal_train2014_with_classes_poly.json\n"
     ]
    }
   ],
   "source": [
    "# Register my amodal datasets \n",
    "from detectron2.data.datasets import register_coco_instances\n",
    "from detectron2.data import MetadataCatalog\n",
    "register_coco_instances(\"amodal_coco_train\", {}, \"datasets/coco/annotations/COCO_amodal_train2014_with_classes_poly.json\", \"datasets/coco/train2014\")\n",
    "# Prepare test datasets \n",
    "from detectron2.data.datasets import register_coco_instances\n",
    "from detectron2.data import MetadataCatalog\n",
    "register_coco_instances(\"amodal_coco_val\", {}, \"datasets/coco/annotations/COCO_amodal_val2014_with_classes_poly.json\", \"datasets/coco/val2014\")\n",
    "from detectron2.data import DatasetCatalog\n",
    "dataset_dicts = DatasetCatalog.get(\"amodal_coco_train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " # Run pre-trained model on this amodal image \n",
    "cfg = get_cfg()\n",
    "# add project-specific config (e.g., TensorMask) here if you're not running a model in detectron2's core library\n",
    "cfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\n",
    "cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model\n",
    "# Find a model from detectron2's model zoo. You can use the https://dl.fbaipublicfiles... url as well\n",
    "cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")\n",
    "from detectron2.data import DatasetMapper\n",
    "mapper = DatasetMapper(cfg,is_train=True)\n",
    "mydict = mapper(dataset_dicts[3])\n",
    "boxes = mydict['instances'].gt_boxes.tensor\n",
    "print(mydict['instances'].gt_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## To verify the data loading is correct, let's visualize the annotations of randomly selected samples in the training set:\n",
    "import random\n",
    "from detectron2.data import DatasetCatalog\n",
    "from detectron2.utils.visualizer import Visualizer\n",
    "dataset_dicts = DatasetCatalog.get(\"amodal_train\")\n",
    "for d in random.sample(dataset_dicts, 1):\n",
    "    img = cv2.imread(d[\"file_name\"])\n",
    "    visualizer = Visualizer(img[:, :, ::-1], metadata=MetadataCatalog.get(\"amodal_train\"), scale=0.5)\n",
    "    vis = visualizer.draw_dataset_dict(d)\n",
    "    imshow(vis.get_image()[:, :, ::-1])\n",
    "    predictor = DefaultPredictor(cfg)\n",
    "    outputs = predictor(img)\n",
    "    # We can use `Visualizer` to draw the predictions on the image.\n",
    "    v = Visualizer(img[:, :, ::-1], MetadataCatalog.get(\"amodal_train\"), scale=1.2)\n",
    "    v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n",
    "    imshow(v.get_image()[:, :, ::-1])\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[02/29 01:22:38 d2.engine.defaults]: \u001b[0mModel:\n",
      "GeneralizedRCNN(\n",
      "  (backbone): FPN(\n",
      "    (fpn_lateral2): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (fpn_output2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (fpn_lateral3): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (fpn_output3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (fpn_lateral4): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (fpn_output4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (fpn_lateral5): Conv2d(2048, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (fpn_output5): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (top_block): LastLevelMaxPool()\n",
      "    (bottom_up): ResNet(\n",
      "      (stem): BasicStem(\n",
      "        (conv1): Conv2d(\n",
      "          3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False\n",
      "          (norm): FrozenBatchNorm2d(num_features=64, eps=1e-05)\n",
      "        )\n",
      "      )\n",
      "      (res2): Sequential(\n",
      "        (0): BottleneckBlock(\n",
      "          (shortcut): Conv2d(\n",
      "            64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv1): Conv2d(\n",
      "            64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=64, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=64, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (1): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=64, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=64, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (2): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=64, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=64, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (res3): Sequential(\n",
      "        (0): BottleneckBlock(\n",
      "          (shortcut): Conv2d(\n",
      "            256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "          (conv1): Conv2d(\n",
      "            256, 128, kernel_size=(1, 1), stride=(2, 2), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=128, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=128, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (1): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=128, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=128, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (2): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=128, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=128, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (3): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=128, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=128, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (res4): Sequential(\n",
      "        (0): BottleneckBlock(\n",
      "          (shortcut): Conv2d(\n",
      "            512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)\n",
      "          )\n",
      "          (conv1): Conv2d(\n",
      "            512, 256, kernel_size=(1, 1), stride=(2, 2), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (1): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (2): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (3): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (4): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (5): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=256, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=1024, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (res5): Sequential(\n",
      "        (0): BottleneckBlock(\n",
      "          (shortcut): Conv2d(\n",
      "            1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)\n",
      "          )\n",
      "          (conv1): Conv2d(\n",
      "            1024, 512, kernel_size=(1, 1), stride=(2, 2), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (1): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "        (2): BottleneckBlock(\n",
      "          (conv1): Conv2d(\n",
      "            2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "          (conv2): Conv2d(\n",
      "            512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=512, eps=1e-05)\n",
      "          )\n",
      "          (conv3): Conv2d(\n",
      "            512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
      "            (norm): FrozenBatchNorm2d(num_features=2048, eps=1e-05)\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (proposal_generator): RPN(\n",
      "    (anchor_generator): DefaultAnchorGenerator(\n",
      "      (cell_anchors): BufferList()\n",
      "    )\n",
      "    (rpn_head): StandardRPNHead(\n",
      "      (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (objectness_logits): Conv2d(256, 3, kernel_size=(1, 1), stride=(1, 1))\n",
      "      (anchor_deltas): Conv2d(256, 12, kernel_size=(1, 1), stride=(1, 1))\n",
      "    )\n",
      "  )\n",
      "  (roi_heads): StandardROIHeads(\n",
      "    (box_pooler): ROIPooler(\n",
      "      (level_poolers): ModuleList(\n",
      "        (0): ROIAlign(output_size=(7, 7), spatial_scale=0.25, sampling_ratio=0, aligned=True)\n",
      "        (1): ROIAlign(output_size=(7, 7), spatial_scale=0.125, sampling_ratio=0, aligned=True)\n",
      "        (2): ROIAlign(output_size=(7, 7), spatial_scale=0.0625, sampling_ratio=0, aligned=True)\n",
      "        (3): ROIAlign(output_size=(7, 7), spatial_scale=0.03125, sampling_ratio=0, aligned=True)\n",
      "      )\n",
      "    )\n",
      "    (box_head): FastRCNNConvFCHead(\n",
      "      (fc1): Linear(in_features=12544, out_features=1024, bias=True)\n",
      "      (fc2): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "    )\n",
      "    (box_predictor): FastRCNNOutputLayers(\n",
      "      (cls_score): Linear(in_features=1024, out_features=81, bias=True)\n",
      "      (bbox_pred): Linear(in_features=1024, out_features=320, bias=True)\n",
      "    )\n",
      "    (mask_pooler): ROIPooler(\n",
      "      (level_poolers): ModuleList(\n",
      "        (0): ROIAlign(output_size=(14, 14), spatial_scale=0.25, sampling_ratio=0, aligned=True)\n",
      "        (1): ROIAlign(output_size=(14, 14), spatial_scale=0.125, sampling_ratio=0, aligned=True)\n",
      "        (2): ROIAlign(output_size=(14, 14), spatial_scale=0.0625, sampling_ratio=0, aligned=True)\n",
      "        (3): ROIAlign(output_size=(14, 14), spatial_scale=0.03125, sampling_ratio=0, aligned=True)\n",
      "      )\n",
      "    )\n",
      "    (mask_head): MaskRCNNConvUpsampleHead(\n",
      "      (mask_fcn1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (mask_fcn2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (mask_fcn3): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (mask_fcn4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "      (deconv): ConvTranspose2d(256, 256, kernel_size=(2, 2), stride=(2, 2))\n",
      "      (predictor): Conv2d(256, 80, kernel_size=(1, 1), stride=(1, 1))\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[02/29 01:22:39 d2.data.datasets.coco]: \u001b[0mLoaded 2276 images in COCO format from datasets/coco/annotations/COCO_amodal_train2014_with_classes_poly.json\n",
      "\u001b[32m[02/29 01:22:39 d2.data.build]: \u001b[0mRemoved 0 images with no usable annotations. 2276 images left.\n",
      "\u001b[32m[02/29 01:22:39 d2.data.build]: \u001b[0mDistribution of instances among all 80 categories:\n",
      "\u001b[36m|   category    | #instances   |   category   | #instances   |   category    | #instances   |\n",
      "|:-------------:|:-------------|:------------:|:-------------|:-------------:|:-------------|\n",
      "|    person     | 2311         |   bicycle    | 21           |      car      | 279          |\n",
      "|  motorcycle   | 47           |   airplane   | 65           |      bus      | 92           |\n",
      "|     train     | 60           |    truck     | 115          |     boat      | 79           |\n",
      "| traffic light | 13           | fire hydrant | 35           |   stop sign   | 12           |\n",
      "| parking meter | 10           |    bench     | 48           |     bird      | 57           |\n",
      "|      cat      | 94           |     dog      | 100          |     horse     | 57           |\n",
      "|     sheep     | 102          |     cow      | 91           |   elephant    | 80           |\n",
      "|     bear      | 28           |    zebra     | 84           |    giraffe    | 90           |\n",
      "|   backpack    | 14           |   umbrella   | 74           |    handbag    | 26           |\n",
      "|      tie      | 9            |   suitcase   | 77           |    frisbee    | 29           |\n",
      "|     skis      | 13           |  snowboard   | 21           |  sports ball  | 36           |\n",
      "|     kite      | 49           | baseball bat | 33           | baseball gl.. | 5            |\n",
      "|  skateboard   | 41           |  surfboard   | 37           | tennis racket | 49           |\n",
      "|    bottle     | 222          |  wine glass  | 70           |      cup      | 215          |\n",
      "|     fork      | 25           |    knife     | 69           |     spoon     | 33           |\n",
      "|     bowl      | 161          |    banana    | 26           |     apple     | 34           |\n",
      "|   sandwich    | 33           |    orange    | 31           |   broccoli    | 9            |\n",
      "|    carrot     | 19           |   hot dog    | 27           |     pizza     | 60           |\n",
      "|     donut     | 70           |     cake     | 76           |     chair     | 195          |\n",
      "|     couch     | 55           | potted plant | 12           |      bed      | 26           |\n",
      "| dining table  | 34           |    toilet    | 80           |      tv       | 91           |\n",
      "|    laptop     | 74           |    mouse     | 25           |    remote     | 28           |\n",
      "|   keyboard    | 36           |  cell phone  | 38           |   microwave   | 33           |\n",
      "|     oven      | 23           |   toaster    | 5            |     sink      | 53           |\n",
      "| refrigerator  | 63           |     book     | 59           |     clock     | 39           |\n",
      "|     vase      | 45           |   scissors   | 8            |  teddy bear   | 68           |\n",
      "|  hair drier   | 0            |  toothbrush  | 10           |               |              |\n",
      "|     total     | 6763         |              |              |               |              |\u001b[0m\n",
      "\u001b[32m[02/29 01:22:39 d2.data.detection_utils]: \u001b[0mTransformGens used in training: [ResizeShortestEdge(short_edge_length=(640, 672, 704, 736, 768, 800), max_size=1333, sample_style='choice'), RandomFlip()]\n",
      "\u001b[32m[02/29 01:22:39 d2.data.build]: \u001b[0mUsing training sampler TrainingSampler\n",
      "\u001b[32m[02/29 01:22:39 d2.engine.train_loop]: \u001b[0mStarting training from iteration 0\n",
      "\u001b[32m[02/29 01:22:45 d2.utils.events]: \u001b[0meta: 0:06:38  iter: 19  total_loss: 0.572  loss_cls: 0.189  loss_box_reg: 0.126  loss_mask: 0.252  loss_rpn_cls: 0.022  loss_rpn_loc: 0.013  time: 0.2693  data_time: 0.0101  lr: 0.000010  max_mem: 2733M\n",
      "\u001b[32m[02/29 01:22:51 d2.utils.events]: \u001b[0meta: 0:06:32  iter: 39  total_loss: 0.524  loss_cls: 0.160  loss_box_reg: 0.079  loss_mask: 0.206  loss_rpn_cls: 0.018  loss_rpn_loc: 0.011  time: 0.2725  data_time: 0.0149  lr: 0.000020  max_mem: 2733M\n",
      "\u001b[32m[02/29 01:22:57 d2.utils.events]: \u001b[0meta: 0:06:27  iter: 59  total_loss: 0.547  loss_cls: 0.165  loss_box_reg: 0.099  loss_mask: 0.264  loss_rpn_cls: 0.011  loss_rpn_loc: 0.010  time: 0.2781  data_time: 0.0152  lr: 0.000030  max_mem: 2733M\n",
      "\u001b[32m[02/29 01:23:03 d2.utils.events]: \u001b[0meta: 0:06:22  iter: 79  total_loss: 0.518  loss_cls: 0.159  loss_box_reg: 0.075  loss_mask: 0.227  loss_rpn_cls: 0.029  loss_rpn_loc: 0.013  time: 0.2773  data_time: 0.0176  lr: 0.000040  max_mem: 2751M\n",
      "\u001b[32m[02/29 01:23:09 d2.utils.events]: \u001b[0meta: 0:06:16  iter: 99  total_loss: 0.418  loss_cls: 0.133  loss_box_reg: 0.067  loss_mask: 0.171  loss_rpn_cls: 0.019  loss_rpn_loc: 0.008  time: 0.2764  data_time: 0.0140  lr: 0.000050  max_mem: 2751M\n",
      "\u001b[32m[02/29 01:23:14 d2.utils.events]: \u001b[0meta: 0:06:10  iter: 119  total_loss: 0.604  loss_cls: 0.124  loss_box_reg: 0.084  loss_mask: 0.297  loss_rpn_cls: 0.025  loss_rpn_loc: 0.015  time: 0.2755  data_time: 0.0135  lr: 0.000060  max_mem: 2751M\n",
      "\u001b[32m[02/29 01:23:20 d2.utils.events]: \u001b[0meta: 0:06:02  iter: 139  total_loss: 0.574  loss_cls: 0.146  loss_box_reg: 0.090  loss_mask: 0.243  loss_rpn_cls: 0.023  loss_rpn_loc: 0.014  time: 0.2738  data_time: 0.0065  lr: 0.000070  max_mem: 2751M\n",
      "\u001b[32m[02/29 01:23:26 d2.utils.events]: \u001b[0meta: 0:05:58  iter: 159  total_loss: 0.636  loss_cls: 0.121  loss_box_reg: 0.112  loss_mask: 0.290  loss_rpn_cls: 0.010  loss_rpn_loc: 0.017  time: 0.2760  data_time: 0.0193  lr: 0.000080  max_mem: 2751M\n",
      "\u001b[32m[02/29 01:23:32 d2.utils.events]: \u001b[0meta: 0:05:54  iter: 179  total_loss: 0.500  loss_cls: 0.148  loss_box_reg: 0.085  loss_mask: 0.238  loss_rpn_cls: 0.028  loss_rpn_loc: 0.016  time: 0.2771  data_time: 0.0209  lr: 0.000090  max_mem: 2751M\n",
      "\u001b[32m[02/29 01:23:39 d2.utils.events]: \u001b[0meta: 0:05:49  iter: 199  total_loss: 0.413  loss_cls: 0.119  loss_box_reg: 0.081  loss_mask: 0.179  loss_rpn_cls: 0.015  loss_rpn_loc: 0.012  time: 0.2784  data_time: 0.0175  lr: 0.000100  max_mem: 2765M\n",
      "\u001b[32m[02/29 01:23:45 d2.utils.events]: \u001b[0meta: 0:05:43  iter: 219  total_loss: 0.461  loss_cls: 0.127  loss_box_reg: 0.090  loss_mask: 0.223  loss_rpn_cls: 0.016  loss_rpn_loc: 0.010  time: 0.2784  data_time: 0.0126  lr: 0.000110  max_mem: 2765M\n",
      "\u001b[32m[02/29 01:23:51 d2.utils.events]: \u001b[0meta: 0:05:38  iter: 239  total_loss: 0.426  loss_cls: 0.092  loss_box_reg: 0.072  loss_mask: 0.253  loss_rpn_cls: 0.013  loss_rpn_loc: 0.007  time: 0.2784  data_time: 0.0162  lr: 0.000120  max_mem: 2765M\n",
      "\u001b[32m[02/29 01:23:57 d2.utils.events]: \u001b[0meta: 0:05:33  iter: 259  total_loss: 0.625  loss_cls: 0.164  loss_box_reg: 0.125  loss_mask: 0.246  loss_rpn_cls: 0.016  loss_rpn_loc: 0.014  time: 0.2791  data_time: 0.0120  lr: 0.000130  max_mem: 2765M\n",
      "\u001b[32m[02/29 01:24:03 d2.utils.events]: \u001b[0meta: 0:05:28  iter: 279  total_loss: 0.460  loss_cls: 0.110  loss_box_reg: 0.096  loss_mask: 0.200  loss_rpn_cls: 0.011  loss_rpn_loc: 0.009  time: 0.2794  data_time: 0.0177  lr: 0.000140  max_mem: 2765M\n",
      "\u001b[32m[02/29 01:24:09 d2.utils.events]: \u001b[0meta: 0:05:23  iter: 299  total_loss: 0.445  loss_cls: 0.101  loss_box_reg: 0.085  loss_mask: 0.235  loss_rpn_cls: 0.014  loss_rpn_loc: 0.010  time: 0.2800  data_time: 0.0119  lr: 0.000150  max_mem: 2765M\n",
      "\u001b[32m[02/29 01:24:16 d2.utils.events]: \u001b[0meta: 0:05:17  iter: 319  total_loss: 0.522  loss_cls: 0.127  loss_box_reg: 0.102  loss_mask: 0.222  loss_rpn_cls: 0.010  loss_rpn_loc: 0.013  time: 0.2804  data_time: 0.0115  lr: 0.000160  max_mem: 2821M\n",
      "\u001b[32m[02/29 01:24:22 d2.utils.events]: \u001b[0meta: 0:05:12  iter: 339  total_loss: 0.530  loss_cls: 0.124  loss_box_reg: 0.098  loss_mask: 0.235  loss_rpn_cls: 0.010  loss_rpn_loc: 0.010  time: 0.2801  data_time: 0.0154  lr: 0.000170  max_mem: 2821M\n",
      "\u001b[32m[02/29 01:24:28 d2.utils.events]: \u001b[0meta: 0:05:07  iter: 359  total_loss: 0.501  loss_cls: 0.142  loss_box_reg: 0.093  loss_mask: 0.206  loss_rpn_cls: 0.017  loss_rpn_loc: 0.015  time: 0.2805  data_time: 0.0148  lr: 0.000180  max_mem: 2821M\n",
      "\u001b[32m[02/29 01:24:34 d2.utils.events]: \u001b[0meta: 0:05:01  iter: 379  total_loss: 0.452  loss_cls: 0.132  loss_box_reg: 0.102  loss_mask: 0.223  loss_rpn_cls: 0.012  loss_rpn_loc: 0.013  time: 0.2805  data_time: 0.0157  lr: 0.000190  max_mem: 2821M\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[02/29 01:24:40 d2.utils.events]: \u001b[0meta: 0:04:55  iter: 399  total_loss: 0.463  loss_cls: 0.120  loss_box_reg: 0.104  loss_mask: 0.178  loss_rpn_cls: 0.025  loss_rpn_loc: 0.010  time: 0.2799  data_time: 0.0156  lr: 0.000200  max_mem: 2821M\n",
      "\u001b[32m[02/29 01:24:46 d2.utils.events]: \u001b[0meta: 0:04:50  iter: 419  total_loss: 0.429  loss_cls: 0.115  loss_box_reg: 0.081  loss_mask: 0.179  loss_rpn_cls: 0.013  loss_rpn_loc: 0.012  time: 0.2797  data_time: 0.0129  lr: 0.000210  max_mem: 2821M\n",
      "\u001b[32m[02/29 01:24:52 d2.utils.events]: \u001b[0meta: 0:04:44  iter: 439  total_loss: 0.400  loss_cls: 0.100  loss_box_reg: 0.086  loss_mask: 0.169  loss_rpn_cls: 0.006  loss_rpn_loc: 0.011  time: 0.2795  data_time: 0.0130  lr: 0.000220  max_mem: 2821M\n",
      "\u001b[32m[02/29 01:24:58 d2.utils.events]: \u001b[0meta: 0:04:39  iter: 459  total_loss: 0.428  loss_cls: 0.125  loss_box_reg: 0.098  loss_mask: 0.199  loss_rpn_cls: 0.009  loss_rpn_loc: 0.010  time: 0.2802  data_time: 0.0155  lr: 0.000230  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:25:04 d2.utils.events]: \u001b[0meta: 0:04:34  iter: 479  total_loss: 0.387  loss_cls: 0.104  loss_box_reg: 0.078  loss_mask: 0.161  loss_rpn_cls: 0.010  loss_rpn_loc: 0.011  time: 0.2803  data_time: 0.0118  lr: 0.000240  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:25:11 d2.utils.events]: \u001b[0meta: 0:04:28  iter: 499  total_loss: 0.376  loss_cls: 0.121  loss_box_reg: 0.100  loss_mask: 0.148  loss_rpn_cls: 0.008  loss_rpn_loc: 0.011  time: 0.2805  data_time: 0.0167  lr: 0.000250  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:25:17 d2.utils.events]: \u001b[0meta: 0:04:23  iter: 519  total_loss: 0.355  loss_cls: 0.079  loss_box_reg: 0.067  loss_mask: 0.175  loss_rpn_cls: 0.007  loss_rpn_loc: 0.010  time: 0.2808  data_time: 0.0210  lr: 0.000260  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:25:23 d2.utils.events]: \u001b[0meta: 0:04:18  iter: 539  total_loss: 0.458  loss_cls: 0.116  loss_box_reg: 0.098  loss_mask: 0.186  loss_rpn_cls: 0.009  loss_rpn_loc: 0.013  time: 0.2814  data_time: 0.0181  lr: 0.000270  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:25:29 d2.utils.events]: \u001b[0meta: 0:04:12  iter: 559  total_loss: 0.470  loss_cls: 0.116  loss_box_reg: 0.083  loss_mask: 0.181  loss_rpn_cls: 0.008  loss_rpn_loc: 0.011  time: 0.2814  data_time: 0.0122  lr: 0.000280  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:25:36 d2.utils.events]: \u001b[0meta: 0:04:07  iter: 579  total_loss: 0.582  loss_cls: 0.171  loss_box_reg: 0.137  loss_mask: 0.228  loss_rpn_cls: 0.016  loss_rpn_loc: 0.011  time: 0.2818  data_time: 0.0182  lr: 0.000290  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:25:42 d2.utils.events]: \u001b[0meta: 0:04:02  iter: 599  total_loss: 0.410  loss_cls: 0.097  loss_box_reg: 0.077  loss_mask: 0.159  loss_rpn_cls: 0.010  loss_rpn_loc: 0.009  time: 0.2818  data_time: 0.0129  lr: 0.000300  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:25:48 d2.utils.events]: \u001b[0meta: 0:03:57  iter: 619  total_loss: 0.436  loss_cls: 0.108  loss_box_reg: 0.107  loss_mask: 0.179  loss_rpn_cls: 0.008  loss_rpn_loc: 0.014  time: 0.2824  data_time: 0.0177  lr: 0.000310  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:25:55 d2.utils.events]: \u001b[0meta: 0:03:51  iter: 639  total_loss: 0.449  loss_cls: 0.095  loss_box_reg: 0.076  loss_mask: 0.218  loss_rpn_cls: 0.008  loss_rpn_loc: 0.010  time: 0.2826  data_time: 0.0185  lr: 0.000320  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:26:01 d2.utils.events]: \u001b[0meta: 0:03:46  iter: 659  total_loss: 0.353  loss_cls: 0.099  loss_box_reg: 0.082  loss_mask: 0.158  loss_rpn_cls: 0.011  loss_rpn_loc: 0.008  time: 0.2824  data_time: 0.0134  lr: 0.000330  max_mem: 2850M\n",
      "\u001b[32m[02/29 01:26:07 d2.utils.events]: \u001b[0meta: 0:03:41  iter: 679  total_loss: 0.393  loss_cls: 0.104  loss_box_reg: 0.098  loss_mask: 0.195  loss_rpn_cls: 0.010  loss_rpn_loc: 0.012  time: 0.2826  data_time: 0.0121  lr: 0.000340  max_mem: 2897M\n",
      "\u001b[32m[02/29 01:26:13 d2.utils.events]: \u001b[0meta: 0:03:35  iter: 699  total_loss: 0.377  loss_cls: 0.103  loss_box_reg: 0.069  loss_mask: 0.172  loss_rpn_cls: 0.009  loss_rpn_loc: 0.008  time: 0.2826  data_time: 0.0144  lr: 0.000350  max_mem: 2968M\n",
      "\u001b[32m[02/29 01:26:19 d2.utils.events]: \u001b[0meta: 0:03:30  iter: 719  total_loss: 0.293  loss_cls: 0.071  loss_box_reg: 0.062  loss_mask: 0.135  loss_rpn_cls: 0.003  loss_rpn_loc: 0.006  time: 0.2824  data_time: 0.0112  lr: 0.000360  max_mem: 2968M\n",
      "\u001b[32m[02/29 01:26:25 d2.utils.events]: \u001b[0meta: 0:03:24  iter: 739  total_loss: 0.482  loss_cls: 0.120  loss_box_reg: 0.106  loss_mask: 0.191  loss_rpn_cls: 0.006  loss_rpn_loc: 0.009  time: 0.2824  data_time: 0.0163  lr: 0.000370  max_mem: 2968M\n",
      "\u001b[32m[02/29 01:26:32 d2.utils.events]: \u001b[0meta: 0:03:19  iter: 759  total_loss: 0.349  loss_cls: 0.086  loss_box_reg: 0.070  loss_mask: 0.143  loss_rpn_cls: 0.007  loss_rpn_loc: 0.010  time: 0.2828  data_time: 0.0228  lr: 0.000380  max_mem: 2968M\n",
      "\u001b[32m[02/29 01:26:38 d2.utils.events]: \u001b[0meta: 0:03:14  iter: 779  total_loss: 0.401  loss_cls: 0.088  loss_box_reg: 0.086  loss_mask: 0.189  loss_rpn_cls: 0.004  loss_rpn_loc: 0.010  time: 0.2828  data_time: 0.0151  lr: 0.000390  max_mem: 2968M\n",
      "\u001b[32m[02/29 01:26:44 d2.utils.events]: \u001b[0meta: 0:03:08  iter: 799  total_loss: 0.501  loss_cls: 0.153  loss_box_reg: 0.102  loss_mask: 0.227  loss_rpn_cls: 0.009  loss_rpn_loc: 0.017  time: 0.2829  data_time: 0.0212  lr: 0.000400  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:26:51 d2.utils.events]: \u001b[0meta: 0:03:03  iter: 819  total_loss: 0.412  loss_cls: 0.099  loss_box_reg: 0.106  loss_mask: 0.188  loss_rpn_cls: 0.005  loss_rpn_loc: 0.011  time: 0.2834  data_time: 0.0176  lr: 0.000410  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:26:57 d2.utils.events]: \u001b[0meta: 0:02:58  iter: 839  total_loss: 0.475  loss_cls: 0.116  loss_box_reg: 0.124  loss_mask: 0.192  loss_rpn_cls: 0.011  loss_rpn_loc: 0.012  time: 0.2834  data_time: 0.0140  lr: 0.000420  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:27:03 d2.utils.events]: \u001b[0meta: 0:02:52  iter: 859  total_loss: 0.342  loss_cls: 0.082  loss_box_reg: 0.095  loss_mask: 0.156  loss_rpn_cls: 0.005  loss_rpn_loc: 0.009  time: 0.2832  data_time: 0.0163  lr: 0.000430  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:27:09 d2.utils.events]: \u001b[0meta: 0:02:47  iter: 879  total_loss: 0.418  loss_cls: 0.118  loss_box_reg: 0.093  loss_mask: 0.169  loss_rpn_cls: 0.011  loss_rpn_loc: 0.011  time: 0.2832  data_time: 0.0132  lr: 0.000440  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:27:15 d2.utils.events]: \u001b[0meta: 0:02:42  iter: 899  total_loss: 0.524  loss_cls: 0.144  loss_box_reg: 0.131  loss_mask: 0.216  loss_rpn_cls: 0.007  loss_rpn_loc: 0.012  time: 0.2832  data_time: 0.0125  lr: 0.000450  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:27:21 d2.utils.events]: \u001b[0meta: 0:02:36  iter: 919  total_loss: 0.353  loss_cls: 0.100  loss_box_reg: 0.071  loss_mask: 0.172  loss_rpn_cls: 0.011  loss_rpn_loc: 0.012  time: 0.2833  data_time: 0.0156  lr: 0.000460  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:27:27 d2.utils.events]: \u001b[0meta: 0:02:31  iter: 939  total_loss: 0.518  loss_cls: 0.095  loss_box_reg: 0.093  loss_mask: 0.199  loss_rpn_cls: 0.009  loss_rpn_loc: 0.011  time: 0.2835  data_time: 0.0121  lr: 0.000470  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:27:34 d2.utils.events]: \u001b[0meta: 0:02:25  iter: 959  total_loss: 0.384  loss_cls: 0.096  loss_box_reg: 0.101  loss_mask: 0.161  loss_rpn_cls: 0.005  loss_rpn_loc: 0.008  time: 0.2836  data_time: 0.0199  lr: 0.000480  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:27:40 d2.utils.events]: \u001b[0meta: 0:02:20  iter: 979  total_loss: 0.433  loss_cls: 0.113  loss_box_reg: 0.091  loss_mask: 0.182  loss_rpn_cls: 0.012  loss_rpn_loc: 0.009  time: 0.2839  data_time: 0.0163  lr: 0.000490  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:27:46 d2.utils.events]: \u001b[0meta: 0:02:15  iter: 999  total_loss: 0.382  loss_cls: 0.123  loss_box_reg: 0.087  loss_mask: 0.200  loss_rpn_cls: 0.006  loss_rpn_loc: 0.008  time: 0.2839  data_time: 0.0168  lr: 0.000500  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:27:52 d2.utils.events]: \u001b[0meta: 0:02:09  iter: 1019  total_loss: 0.526  loss_cls: 0.142  loss_box_reg: 0.126  loss_mask: 0.229  loss_rpn_cls: 0.010  loss_rpn_loc: 0.015  time: 0.2839  data_time: 0.0107  lr: 0.000500  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:27:59 d2.utils.events]: \u001b[0meta: 0:02:04  iter: 1039  total_loss: 0.383  loss_cls: 0.102  loss_box_reg: 0.089  loss_mask: 0.159  loss_rpn_cls: 0.010  loss_rpn_loc: 0.008  time: 0.2839  data_time: 0.0034  lr: 0.000500  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:28:05 d2.utils.events]: \u001b[0meta: 0:01:59  iter: 1059  total_loss: 0.445  loss_cls: 0.119  loss_box_reg: 0.100  loss_mask: 0.177  loss_rpn_cls: 0.007  loss_rpn_loc: 0.010  time: 0.2841  data_time: 0.0166  lr: 0.000500  max_mem: 3003M\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[02/29 01:28:11 d2.utils.events]: \u001b[0meta: 0:01:53  iter: 1079  total_loss: 0.357  loss_cls: 0.106  loss_box_reg: 0.089  loss_mask: 0.175  loss_rpn_cls: 0.006  loss_rpn_loc: 0.010  time: 0.2841  data_time: 0.0115  lr: 0.000500  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:28:17 d2.utils.events]: \u001b[0meta: 0:01:48  iter: 1099  total_loss: 0.496  loss_cls: 0.141  loss_box_reg: 0.138  loss_mask: 0.194  loss_rpn_cls: 0.012  loss_rpn_loc: 0.014  time: 0.2841  data_time: 0.0129  lr: 0.000500  max_mem: 3003M\n",
      "\u001b[32m[02/29 01:28:24 d2.utils.events]: \u001b[0meta: 0:01:43  iter: 1119  total_loss: 0.419  loss_cls: 0.101  loss_box_reg: 0.100  loss_mask: 0.172  loss_rpn_cls: 0.007  loss_rpn_loc: 0.011  time: 0.2843  data_time: 0.0158  lr: 0.000500  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:28:30 d2.utils.events]: \u001b[0meta: 0:01:37  iter: 1139  total_loss: 0.351  loss_cls: 0.084  loss_box_reg: 0.081  loss_mask: 0.169  loss_rpn_cls: 0.006  loss_rpn_loc: 0.009  time: 0.2844  data_time: 0.0160  lr: 0.000500  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:28:36 d2.utils.events]: \u001b[0meta: 0:01:32  iter: 1159  total_loss: 0.375  loss_cls: 0.085  loss_box_reg: 0.092  loss_mask: 0.164  loss_rpn_cls: 0.002  loss_rpn_loc: 0.007  time: 0.2847  data_time: 0.0233  lr: 0.000500  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:28:43 d2.utils.events]: \u001b[0meta: 0:01:26  iter: 1179  total_loss: 0.486  loss_cls: 0.117  loss_box_reg: 0.122  loss_mask: 0.194  loss_rpn_cls: 0.006  loss_rpn_loc: 0.011  time: 0.2848  data_time: 0.0122  lr: 0.000500  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:28:49 d2.utils.events]: \u001b[0meta: 0:01:21  iter: 1199  total_loss: 0.416  loss_cls: 0.111  loss_box_reg: 0.093  loss_mask: 0.182  loss_rpn_cls: 0.007  loss_rpn_loc: 0.008  time: 0.2849  data_time: 0.0136  lr: 0.000500  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:28:55 d2.utils.events]: \u001b[0meta: 0:01:16  iter: 1219  total_loss: 0.374  loss_cls: 0.105  loss_box_reg: 0.094  loss_mask: 0.185  loss_rpn_cls: 0.007  loss_rpn_loc: 0.011  time: 0.2849  data_time: 0.0139  lr: 0.000050  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:29:01 d2.utils.events]: \u001b[0meta: 0:01:10  iter: 1239  total_loss: 0.314  loss_cls: 0.070  loss_box_reg: 0.074  loss_mask: 0.143  loss_rpn_cls: 0.004  loss_rpn_loc: 0.009  time: 0.2849  data_time: 0.0165  lr: 0.000050  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:29:08 d2.utils.events]: \u001b[0meta: 0:01:05  iter: 1259  total_loss: 0.406  loss_cls: 0.112  loss_box_reg: 0.095  loss_mask: 0.168  loss_rpn_cls: 0.007  loss_rpn_loc: 0.013  time: 0.2850  data_time: 0.0142  lr: 0.000050  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:29:14 d2.utils.events]: \u001b[0meta: 0:01:00  iter: 1279  total_loss: 0.513  loss_cls: 0.115  loss_box_reg: 0.128  loss_mask: 0.201  loss_rpn_cls: 0.009  loss_rpn_loc: 0.010  time: 0.2851  data_time: 0.0157  lr: 0.000050  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:29:21 d2.utils.events]: \u001b[0meta: 0:00:54  iter: 1299  total_loss: 0.462  loss_cls: 0.133  loss_box_reg: 0.133  loss_mask: 0.166  loss_rpn_cls: 0.007  loss_rpn_loc: 0.013  time: 0.2853  data_time: 0.0185  lr: 0.000050  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:29:27 d2.utils.events]: \u001b[0meta: 0:00:49  iter: 1319  total_loss: 0.458  loss_cls: 0.117  loss_box_reg: 0.110  loss_mask: 0.165  loss_rpn_cls: 0.007  loss_rpn_loc: 0.013  time: 0.2853  data_time: 0.0127  lr: 0.000005  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:29:33 d2.utils.events]: \u001b[0meta: 0:00:43  iter: 1339  total_loss: 0.367  loss_cls: 0.087  loss_box_reg: 0.087  loss_mask: 0.154  loss_rpn_cls: 0.004  loss_rpn_loc: 0.008  time: 0.2852  data_time: 0.0135  lr: 0.000005  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:29:39 d2.utils.events]: \u001b[0meta: 0:00:38  iter: 1359  total_loss: 0.384  loss_cls: 0.102  loss_box_reg: 0.095  loss_mask: 0.178  loss_rpn_cls: 0.008  loss_rpn_loc: 0.008  time: 0.2854  data_time: 0.0156  lr: 0.000005  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:29:46 d2.utils.events]: \u001b[0meta: 0:00:32  iter: 1379  total_loss: 0.410  loss_cls: 0.110  loss_box_reg: 0.088  loss_mask: 0.162  loss_rpn_cls: 0.006  loss_rpn_loc: 0.009  time: 0.2854  data_time: 0.0175  lr: 0.000005  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:29:52 d2.utils.events]: \u001b[0meta: 0:00:27  iter: 1399  total_loss: 0.505  loss_cls: 0.118  loss_box_reg: 0.105  loss_mask: 0.207  loss_rpn_cls: 0.007  loss_rpn_loc: 0.013  time: 0.2856  data_time: 0.0143  lr: 0.000005  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:29:58 d2.utils.events]: \u001b[0meta: 0:00:22  iter: 1419  total_loss: 0.398  loss_cls: 0.095  loss_box_reg: 0.120  loss_mask: 0.159  loss_rpn_cls: 0.004  loss_rpn_loc: 0.011  time: 0.2856  data_time: 0.0106  lr: 0.000005  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:30:05 d2.utils.events]: \u001b[0meta: 0:00:16  iter: 1439  total_loss: 0.509  loss_cls: 0.132  loss_box_reg: 0.124  loss_mask: 0.211  loss_rpn_cls: 0.007  loss_rpn_loc: 0.020  time: 0.2859  data_time: 0.0137  lr: 0.000005  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:30:11 d2.utils.events]: \u001b[0meta: 0:00:11  iter: 1459  total_loss: 0.374  loss_cls: 0.103  loss_box_reg: 0.086  loss_mask: 0.161  loss_rpn_cls: 0.007  loss_rpn_loc: 0.009  time: 0.2860  data_time: 0.0177  lr: 0.000005  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:30:17 d2.utils.events]: \u001b[0meta: 0:00:05  iter: 1479  total_loss: 0.340  loss_cls: 0.081  loss_box_reg: 0.081  loss_mask: 0.175  loss_rpn_cls: 0.006  loss_rpn_loc: 0.008  time: 0.2860  data_time: 0.0127  lr: 0.000005  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:30:24 d2.data.datasets.coco]: \u001b[0mLoaded 2276 images in COCO format from datasets/coco/annotations/COCO_amodal_train2014_with_classes_poly.json\n",
      "\u001b[5m\u001b[31mWARNING\u001b[0m \u001b[32m[02/29 01:30:24 d2.engine.defaults]: \u001b[0mNo evaluator found. Use `DefaultTrainer.test(evaluators=)`, or implement its `build_evaluator` method.\n",
      "\u001b[32m[02/29 01:30:24 d2.utils.events]: \u001b[0meta: 0:00:00  iter: 1499  total_loss: 0.402  loss_cls: 0.107  loss_box_reg: 0.091  loss_mask: 0.161  loss_rpn_cls: 0.009  loss_rpn_loc: 0.012  time: 0.2859  data_time: 0.0111  lr: 0.000005  max_mem: 3046M\n",
      "\u001b[32m[02/29 01:30:24 d2.engine.hooks]: \u001b[0mOverall training speed: 1497 iterations in 0:07:08 (0.2861 s / it)\n",
      "\u001b[32m[02/29 01:30:24 d2.engine.hooks]: \u001b[0mTotal training time: 0:07:44 (0:00:36 on hooks)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from detectron2.engine import DefaultTrainer\n",
    "from detectron2.config import get_cfg\n",
    "\n",
    "cfg = get_cfg()\n",
    "cfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\n",
    "cfg.DATASETS.TRAIN = (\"amodal_coco_train\",)\n",
    "cfg.DATASETS.TEST = (\"amodal_coco_train\",)\n",
    "cfg.DATALOADER.NUM_WORKERS = 2\n",
    "cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")  # Let training initialize from model zoo\n",
    "cfg.SOLVER.IMS_PER_BATCH = 2\n",
    "cfg.SOLVER.BASE_LR = 0.0005  # pick a good LR\n",
    "cfg.SOLVER.STEPS = (1200,1300)\n",
    "cfg.SOLVER.MAX_ITER = 1500\n",
    "cfg.VIS_PERIOD = 20\n",
    "cfg.OUTPUT_DIR = \"myAmodalCheckpoint\"\n",
    "os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)\n",
    "trainer = DefaultTrainer(cfg) \n",
    "trainer.resume_or_load(resume=False)\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!tensorboard --logdir=myAmodalCheckpoint --port=6006"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Inference & evaluation using the trained model\n",
    "Now, let's run inference with the trained model on the balloon validation dataset. First, let's create a predictor using the model we just trained:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[02/29 01:30:47 d2.data.datasets.coco]: \u001b[0mLoaded 1223 images in COCO format from datasets/coco/annotations/COCO_amodal_val2014_with_classes_poly.json\n",
      "\u001b[32m[02/29 01:30:47 d2.data.build]: \u001b[0mDistribution of instances among all 80 categories:\n",
      "\u001b[36m|   category    | #instances   |   category   | #instances   |   category    | #instances   |\n",
      "|:-------------:|:-------------|:------------:|:-------------|:-------------:|:-------------|\n",
      "|    person     | 1197         |   bicycle    | 12           |      car      | 154          |\n",
      "|  motorcycle   | 31           |   airplane   | 39           |      bus      | 44           |\n",
      "|     train     | 42           |    truck     | 44           |     boat      | 49           |\n",
      "| traffic light | 7            | fire hydrant | 17           |   stop sign   | 9            |\n",
      "| parking meter | 2            |    bench     | 34           |     bird      | 42           |\n",
      "|      cat      | 42           |     dog      | 62           |     horse     | 48           |\n",
      "|     sheep     | 49           |     cow      | 56           |   elephant    | 43           |\n",
      "|     bear      | 11           |    zebra     | 41           |    giraffe    | 31           |\n",
      "|   backpack    | 12           |   umbrella   | 73           |    handbag    | 24           |\n",
      "|      tie      | 23           |   suitcase   | 42           |    frisbee    | 27           |\n",
      "|     skis      | 9            |  snowboard   | 9            |  sports ball  | 14           |\n",
      "|     kite      | 32           | baseball bat | 18           | baseball gl.. | 12           |\n",
      "|  skateboard   | 16           |  surfboard   | 14           | tennis racket | 38           |\n",
      "|    bottle     | 91           |  wine glass  | 26           |      cup      | 141          |\n",
      "|     fork      | 19           |    knife     | 26           |     spoon     | 22           |\n",
      "|     bowl      | 51           |    banana    | 20           |     apple     | 19           |\n",
      "|   sandwich    | 28           |    orange    | 17           |   broccoli    | 16           |\n",
      "|    carrot     | 10           |   hot dog    | 23           |     pizza     | 60           |\n",
      "|     donut     | 54           |     cake     | 33           |     chair     | 127          |\n",
      "|     couch     | 26           | potted plant | 15           |      bed      | 12           |\n",
      "| dining table  | 32           |    toilet    | 45           |      tv       | 53           |\n",
      "|    laptop     | 52           |    mouse     | 23           |    remote     | 18           |\n",
      "|   keyboard    | 28           |  cell phone  | 31           |   microwave   | 8            |\n",
      "|     oven      | 11           |   toaster    | 2            |     sink      | 24           |\n",
      "| refrigerator  | 13           |     book     | 23           |     clock     | 33           |\n",
      "|     vase      | 42           |   scissors   | 12           |  teddy bear   | 41           |\n",
      "|  hair drier   | 1            |  toothbrush  | 2            |               |              |\n",
      "|     total     | 3799         |              |              |               |              |\u001b[0m\n",
      "\u001b[32m[02/29 01:30:47 d2.evaluation.evaluator]: \u001b[0mStart inference on 1223 images\n",
      "\u001b[32m[02/29 01:30:48 d2.evaluation.evaluator]: \u001b[0mInference done 11/1223. 0.0525 s / img. ETA=0:01:17\n",
      "\u001b[32m[02/29 01:30:53 d2.evaluation.evaluator]: \u001b[0mInference done 93/1223. 0.0529 s / img. ETA=0:01:09\n",
      "\u001b[32m[02/29 01:30:58 d2.evaluation.evaluator]: \u001b[0mInference done 177/1223. 0.0528 s / img. ETA=0:01:03\n",
      "\u001b[32m[02/29 01:31:03 d2.evaluation.evaluator]: \u001b[0mInference done 259/1223. 0.0528 s / img. ETA=0:00:58\n",
      "\u001b[32m[02/29 01:31:08 d2.evaluation.evaluator]: \u001b[0mInference done 341/1223. 0.0528 s / img. ETA=0:00:53\n",
      "\u001b[32m[02/29 01:31:13 d2.evaluation.evaluator]: \u001b[0mInference done 425/1223. 0.0529 s / img. ETA=0:00:48\n",
      "\u001b[32m[02/29 01:31:18 d2.evaluation.evaluator]: \u001b[0mInference done 506/1223. 0.0530 s / img. ETA=0:00:43\n",
      "\u001b[32m[02/29 01:31:23 d2.evaluation.evaluator]: \u001b[0mInference done 587/1223. 0.0531 s / img. ETA=0:00:38\n",
      "\u001b[32m[02/29 01:31:28 d2.evaluation.evaluator]: \u001b[0mInference done 667/1223. 0.0532 s / img. ETA=0:00:34\n",
      "\u001b[32m[02/29 01:31:33 d2.evaluation.evaluator]: \u001b[0mInference done 749/1223. 0.0532 s / img. ETA=0:00:29\n",
      "\u001b[32m[02/29 01:31:38 d2.evaluation.evaluator]: \u001b[0mInference done 830/1223. 0.0533 s / img. ETA=0:00:24\n",
      "\u001b[32m[02/29 01:31:43 d2.evaluation.evaluator]: \u001b[0mInference done 914/1223. 0.0532 s / img. ETA=0:00:18\n",
      "\u001b[32m[02/29 01:31:48 d2.evaluation.evaluator]: \u001b[0mInference done 996/1223. 0.0532 s / img. ETA=0:00:13\n",
      "\u001b[32m[02/29 01:31:53 d2.evaluation.evaluator]: \u001b[0mInference done 1078/1223. 0.0533 s / img. ETA=0:00:08\n",
      "\u001b[32m[02/29 01:31:58 d2.evaluation.evaluator]: \u001b[0mInference done 1158/1223. 0.0533 s / img. ETA=0:00:03\n",
      "\u001b[32m[02/29 01:32:02 d2.evaluation.evaluator]: \u001b[0mTotal inference time: 0:01:14.913446 (0.061505 s / img per device, on 1 devices)\n",
      "\u001b[32m[02/29 01:32:02 d2.evaluation.evaluator]: \u001b[0mTotal inference pure compute time: 0:01:04 (0.053292 s / img per device, on 1 devices)\n",
      "\u001b[32m[02/29 01:32:02 d2.evaluation.coco_evaluation]: \u001b[0mPreparing results for COCO format ...\n",
      "\u001b[32m[02/29 01:32:02 d2.evaluation.coco_evaluation]: \u001b[0mSaving results to myAmodalEvaluation/coco_instances_results.json\n",
      "\u001b[32m[02/29 01:32:03 d2.evaluation.coco_evaluation]: \u001b[0mEvaluating predictions ...\n",
      "Loading and preparing results...\n",
      "DONE (t=0.01s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=4.14s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.11s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.466\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.627\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.522\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.401\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.473\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.523\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.483\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.668\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.679\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.556\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.690\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.698\n",
      "\u001b[32m[02/29 01:32:08 d2.evaluation.coco_evaluation]: \u001b[0mEvaluation results for bbox: \n",
      "|   AP   |  AP50  |  AP75  |  APs   |  APm   |  APl   |\n",
      "|:------:|:------:|:------:|:------:|:------:|:------:|\n",
      "| 46.560 | 62.746 | 52.219 | 40.112 | 47.321 | 52.316 |\n",
      "\u001b[32m[02/29 01:32:08 d2.evaluation.coco_evaluation]: \u001b[0mPer-category bbox AP: \n",
      "| category      | AP     | category     | AP     | category       | AP     |\n",
      "|:--------------|:-------|:-------------|:-------|:---------------|:-------|\n",
      "| person        | 61.326 | bicycle      | 32.132 | car            | 46.626 |\n",
      "| motorcycle    | 44.907 | airplane     | 68.159 | bus            | 67.796 |\n",
      "| train         | 64.561 | truck        | 43.166 | boat           | 29.442 |\n",
      "| traffic light | 7.231  | fire hydrant | 82.215 | stop sign      | 24.427 |\n",
      "| parking meter | 27.970 | bench        | 57.007 | bird           | 43.052 |\n",
      "| cat           | 69.273 | dog          | 64.431 | horse          | 68.976 |\n",
      "| sheep         | 64.263 | cow          | 59.707 | elephant       | 64.337 |\n",
      "| bear          | 76.282 | zebra        | 65.355 | giraffe        | 82.357 |\n",
      "| backpack      | 13.850 | umbrella     | 42.795 | handbag        | 20.611 |\n",
      "| tie           | 21.280 | suitcase     | 49.211 | frisbee        | 74.661 |\n",
      "| skis          | 5.643  | snowboard    | 21.911 | sports ball    | 62.738 |\n",
      "| kite          | 62.318 | baseball bat | 25.082 | baseball glove | 30.649 |\n",
      "| skateboard    | 62.621 | surfboard    | 34.167 | tennis racket  | 68.787 |\n",
      "| bottle        | 55.357 | wine glass   | 63.679 | cup            | 56.317 |\n",
      "| fork          | 40.615 | knife        | 25.766 | spoon          | 40.870 |\n",
      "| bowl          | 26.812 | banana       | 12.094 | apple          | 50.128 |\n",
      "| sandwich      | 44.481 | orange       | 80.326 | broccoli       | 38.592 |\n",
      "| carrot        | 9.826  | hot dog      | 68.617 | pizza          | 62.670 |\n",
      "| donut         | 60.760 | cake         | 54.782 | chair          | 36.550 |\n",
      "| couch         | 48.999 | potted plant | 12.231 | bed            | 44.878 |\n",
      "| dining table  | 8.147  | toilet       | 69.117 | tv             | 64.461 |\n",
      "| laptop        | 74.620 | mouse        | 37.617 | remote         | 31.840 |\n",
      "| keyboard      | 58.133 | cell phone   | 41.887 | microwave      | 40.927 |\n",
      "| oven          | 51.081 | toaster      | 28.663 | sink           | 34.438 |\n",
      "| refrigerator  | 55.512 | book         | 19.388 | clock          | 52.282 |\n",
      "| vase          | 45.514 | scissors     | 45.100 | teddy bear     | 50.566 |\n",
      "| hair drier    | 26.667 | toothbrush   | 43.188 |                |        |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading and preparing results...\n",
      "DONE (t=0.30s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *segm*\n",
      "DONE (t=4.65s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=1.11s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.448\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.630\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.515\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.292\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.430\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.529\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.464\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.636\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.646\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.513\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.654\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.655\n",
      "\u001b[32m[02/29 01:32:14 d2.evaluation.coco_evaluation]: \u001b[0mEvaluation results for segm: \n",
      "|   AP   |  AP50  |  AP75  |  APs   |  APm   |  APl   |\n",
      "|:------:|:------:|:------:|:------:|:------:|:------:|\n",
      "| 44.843 | 63.017 | 51.454 | 29.195 | 42.965 | 52.921 |\n",
      "\u001b[32m[02/29 01:32:14 d2.evaluation.coco_evaluation]: \u001b[0mPer-category segm AP: \n",
      "| category      | AP     | category     | AP     | category       | AP     |\n",
      "|:--------------|:-------|:-------------|:-------|:---------------|:-------|\n",
      "| person        | 55.091 | bicycle      | 18.284 | car            | 46.146 |\n",
      "| motorcycle    | 39.089 | airplane     | 56.948 | bus            | 71.873 |\n",
      "| train         | 68.047 | truck        | 43.659 | boat           | 29.311 |\n",
      "| traffic light | 6.656  | fire hydrant | 74.530 | stop sign      | 42.030 |\n",
      "| parking meter | 26.733 | bench        | 50.299 | bird           | 41.742 |\n",
      "| cat           | 76.351 | dog          | 63.146 | horse          | 55.881 |\n",
      "| sheep         | 60.944 | cow          | 53.977 | elephant       | 57.684 |\n",
      "| bear          | 79.339 | zebra        | 51.550 | giraffe        | 58.265 |\n",
      "| backpack      | 17.219 | umbrella     | 50.533 | handbag        | 18.289 |\n",
      "| tie           | 22.784 | suitcase     | 45.944 | frisbee        | 70.337 |\n",
      "| skis          | 1.197  | snowboard    | 20.321 | sports ball    | 59.681 |\n",
      "| kite          | 54.685 | baseball bat | 26.694 | baseball glove | 32.921 |\n",
      "| skateboard    | 49.931 | surfboard    | 45.030 | tennis racket  | 70.253 |\n",
      "| bottle        | 52.009 | wine glass   | 59.184 | cup            | 59.684 |\n",
      "| fork          | 24.028 | knife        | 24.771 | spoon          | 35.285 |\n",
      "| bowl          | 25.987 | banana       | 9.934  | apple          | 53.505 |\n",
      "| sandwich      | 44.553 | orange       | 80.643 | broccoli       | 39.235 |\n",
      "| carrot        | 11.377 | hot dog      | 65.684 | pizza          | 61.973 |\n",
      "| donut         | 59.561 | cake         | 55.880 | chair          | 27.441 |\n",
      "| couch         | 46.997 | potted plant | 12.456 | bed            | 45.420 |\n",
      "| dining table  | 4.569  | toilet       | 68.313 | tv             | 68.901 |\n",
      "| laptop        | 70.106 | mouse        | 60.468 | remote         | 29.427 |\n",
      "| keyboard      | 63.985 | cell phone   | 43.367 | microwave      | 40.040 |\n",
      "| oven          | 46.566 | toaster      | 28.119 | sink           | 43.150 |\n",
      "| refrigerator  | 50.123 | book         | 17.991 | clock          | 51.938 |\n",
      "| vase          | 48.016 | scissors     | 27.328 | teddy bear     | 49.492 |\n",
      "| hair drier    | 23.333 | toothbrush   | 43.188 |                |        |\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "OrderedDict([('bbox',\n",
       "              {'AP': 46.560225428433654,\n",
       "               'AP50': 62.74554515780276,\n",
       "               'AP75': 52.21944267002813,\n",
       "               'APs': 40.112298672111926,\n",
       "               'APm': 47.32063596316492,\n",
       "               'APl': 52.31635639062691,\n",
       "               'AP-person': 61.32572172291171,\n",
       "               'AP-bicycle': 32.132329831448615,\n",
       "               'AP-car': 46.62611683740921,\n",
       "               'AP-motorcycle': 44.90709937389528,\n",
       "               'AP-airplane': 68.15856635824889,\n",
       "               'AP-bus': 67.79633215906628,\n",
       "               'AP-train': 64.5607911656831,\n",
       "               'AP-truck': 43.16649622935681,\n",
       "               'AP-boat': 29.441652868552648,\n",
       "               'AP-traffic light': 7.231155446371703,\n",
       "               'AP-fire hydrant': 82.214715589206,\n",
       "               'AP-stop sign': 24.427157001414432,\n",
       "               'AP-parking meter': 27.970297029702973,\n",
       "               'AP-bench': 57.007036752995255,\n",
       "               'AP-bird': 43.05151337250012,\n",
       "               'AP-cat': 69.27256394742884,\n",
       "               'AP-dog': 64.43081723439063,\n",
       "               'AP-horse': 68.9760475776164,\n",
       "               'AP-sheep': 64.26343558734628,\n",
       "               'AP-cow': 59.70721839073221,\n",
       "               'AP-elephant': 64.3374691009801,\n",
       "               'AP-bear': 76.28224250996527,\n",
       "               'AP-zebra': 65.35485747071331,\n",
       "               'AP-giraffe': 82.35681729697471,\n",
       "               'AP-backpack': 13.849569530203908,\n",
       "               'AP-umbrella': 42.79541526847595,\n",
       "               'AP-handbag': 20.610998856465425,\n",
       "               'AP-tie': 21.28035513209007,\n",
       "               'AP-suitcase': 49.21123321208623,\n",
       "               'AP-frisbee': 74.66097846581015,\n",
       "               'AP-skis': 5.642564256425641,\n",
       "               'AP-snowboard': 21.91084030678772,\n",
       "               'AP-sports ball': 62.73762376237625,\n",
       "               'AP-kite': 62.318279499675015,\n",
       "               'AP-baseball bat': 25.081577544496742,\n",
       "               'AP-baseball glove': 30.649227107584704,\n",
       "               'AP-skateboard': 62.620664160230135,\n",
       "               'AP-surfboard': 34.1674473329686,\n",
       "               'AP-tennis racket': 68.7865797180767,\n",
       "               'AP-bottle': 55.35713374454361,\n",
       "               'AP-wine glass': 63.67850546213971,\n",
       "               'AP-cup': 56.31665854092365,\n",
       "               'AP-fork': 40.61499290315827,\n",
       "               'AP-knife': 25.7656800347408,\n",
       "               'AP-spoon': 40.8701534734266,\n",
       "               'AP-bowl': 26.811884861205698,\n",
       "               'AP-banana': 12.093734189059088,\n",
       "               'AP-apple': 50.12809619750846,\n",
       "               'AP-sandwich': 44.480731248292045,\n",
       "               'AP-orange': 80.32566113754233,\n",
       "               'AP-broccoli': 38.5917696211658,\n",
       "               'AP-carrot': 9.825553983969826,\n",
       "               'AP-hot dog': 68.61703401211416,\n",
       "               'AP-pizza': 62.66998760906928,\n",
       "               'AP-donut': 60.76012322035578,\n",
       "               'AP-cake': 54.78234457984552,\n",
       "               'AP-chair': 36.54990852316152,\n",
       "               'AP-couch': 48.99909531443997,\n",
       "               'AP-potted plant': 12.23116285577414,\n",
       "               'AP-bed': 44.877555844914525,\n",
       "               'AP-dining table': 8.147071283496823,\n",
       "               'AP-toilet': 69.11659622389831,\n",
       "               'AP-tv': 64.46059515090465,\n",
       "               'AP-laptop': 74.61995881250168,\n",
       "               'AP-mouse': 37.61688531490511,\n",
       "               'AP-remote': 31.840423212828195,\n",
       "               'AP-keyboard': 58.1325526239126,\n",
       "               'AP-cell phone': 41.88684010993033,\n",
       "               'AP-microwave': 40.92661051819468,\n",
       "               'AP-oven': 51.08143935990695,\n",
       "               'AP-toaster': 28.66336633663366,\n",
       "               'AP-sink': 34.438466348769445,\n",
       "               'AP-refrigerator': 55.51243595489937,\n",
       "               'AP-book': 19.388440601289076,\n",
       "               'AP-clock': 52.28240063673911,\n",
       "               'AP-vase': 45.51367451097929,\n",
       "               'AP-scissors': 45.09996525968386,\n",
       "               'AP-teddy bear': 50.565954170656546,\n",
       "               'AP-hair drier': 26.66666666666666,\n",
       "               'AP-toothbrush': 43.18811881188119}),\n",
       "             ('segm',\n",
       "              {'AP': 44.84270163211847,\n",
       "               'AP50': 63.017434924993374,\n",
       "               'AP75': 51.45374807733606,\n",
       "               'APs': 29.19473657985609,\n",
       "               'APm': 42.964600554602185,\n",
       "               'APl': 52.92076856258213,\n",
       "               'AP-person': 55.091189134487685,\n",
       "               'AP-bicycle': 18.28365777754246,\n",
       "               'AP-car': 46.14598217763417,\n",
       "               'AP-motorcycle': 39.08871965206042,\n",
       "               'AP-airplane': 56.94847076061226,\n",
       "               'AP-bus': 71.8731480808581,\n",
       "               'AP-train': 68.0472023984824,\n",
       "               'AP-truck': 43.659207334231546,\n",
       "               'AP-boat': 29.31073880884857,\n",
       "               'AP-traffic light': 6.655665566556655,\n",
       "               'AP-fire hydrant': 74.52950883323626,\n",
       "               'AP-stop sign': 42.02970297029702,\n",
       "               'AP-parking meter': 26.732673267326735,\n",
       "               'AP-bench': 50.29865369953663,\n",
       "               'AP-bird': 41.74174078735655,\n",
       "               'AP-cat': 76.35145579941967,\n",
       "               'AP-dog': 63.14556418889544,\n",
       "               'AP-horse': 55.880751976403594,\n",
       "               'AP-sheep': 60.94421988024925,\n",
       "               'AP-cow': 53.97745318571624,\n",
       "               'AP-elephant': 57.68411471490475,\n",
       "               'AP-bear': 79.33888388838884,\n",
       "               'AP-zebra': 51.55018919444644,\n",
       "               'AP-giraffe': 58.265252175898794,\n",
       "               'AP-backpack': 17.21873735769312,\n",
       "               'AP-umbrella': 50.53253041882427,\n",
       "               'AP-handbag': 18.289103460294772,\n",
       "               'AP-tie': 22.783724991165432,\n",
       "               'AP-suitcase': 45.94426672599332,\n",
       "               'AP-frisbee': 70.3365414366834,\n",
       "               'AP-skis': 1.1971197119711972,\n",
       "               'AP-snowboard': 20.320950966488017,\n",
       "               'AP-sports ball': 59.68121812181217,\n",
       "               'AP-kite': 54.68482833156896,\n",
       "               'AP-baseball bat': 26.69360881373018,\n",
       "               'AP-baseball glove': 32.92062315475245,\n",
       "               'AP-skateboard': 49.93128256854802,\n",
       "               'AP-surfboard': 45.02970263412897,\n",
       "               'AP-tennis racket': 70.25346951294186,\n",
       "               'AP-bottle': 52.00852635519818,\n",
       "               'AP-wine glass': 59.18442284181946,\n",
       "               'AP-cup': 59.68374850071935,\n",
       "               'AP-fork': 24.028354373544193,\n",
       "               'AP-knife': 24.77062128729504,\n",
       "               'AP-spoon': 35.284569459140485,\n",
       "               'AP-bowl': 25.987459076748248,\n",
       "               'AP-banana': 9.933804294232273,\n",
       "               'AP-apple': 53.50530390892726,\n",
       "               'AP-sandwich': 44.55316888489245,\n",
       "               'AP-orange': 80.64257854356865,\n",
       "               'AP-broccoli': 39.23495118742644,\n",
       "               'AP-carrot': 11.376935149483636,\n",
       "               'AP-hot dog': 65.68352552362946,\n",
       "               'AP-pizza': 61.973028249492835,\n",
       "               'AP-donut': 59.56092920305856,\n",
       "               'AP-cake': 55.879544465749596,\n",
       "               'AP-chair': 27.441415588121952,\n",
       "               'AP-couch': 46.99713277359942,\n",
       "               'AP-potted plant': 12.45587929468356,\n",
       "               'AP-bed': 45.41993847027383,\n",
       "               'AP-dining table': 4.569162292564586,\n",
       "               'AP-toilet': 68.31332973508059,\n",
       "               'AP-tv': 68.90074907247369,\n",
       "               'AP-laptop': 70.10580096030701,\n",
       "               'AP-mouse': 60.46756216741844,\n",
       "               'AP-remote': 29.426937163762464,\n",
       "               'AP-keyboard': 63.98464426147501,\n",
       "               'AP-cell phone': 43.36729973129714,\n",
       "               'AP-microwave': 40.039807552183795,\n",
       "               'AP-oven': 46.565573299616716,\n",
       "               'AP-toaster': 28.118811881188122,\n",
       "               'AP-sink': 43.15013691179825,\n",
       "               'AP-refrigerator': 50.1232012304325,\n",
       "               'AP-book': 17.99076155831991,\n",
       "               'AP-clock': 51.938425876652296,\n",
       "               'AP-vase': 48.015728977790786,\n",
       "               'AP-scissors': 27.327545035205276,\n",
       "               'AP-teddy bear': 49.49150882510428,\n",
       "               'AP-hair drier': 23.333333333333332,\n",
       "               'AP-toothbrush': 43.18811881188119})])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  evaluate its performance using AP metric implemented in COCO API.\n",
    "from detectron2.evaluation import COCOEvaluator, inference_on_dataset\n",
    "from detectron2.data import build_detection_test_loader\n",
    "evaluator = COCOEvaluator(\"amodal_coco_val\", cfg, False, output_dir=\"myAmodalEvaluation\")\n",
    "val_loader = build_detection_test_loader(cfg, \"amodal_coco_val\")\n",
    "inference_on_dataset(trainer.model, val_loader, evaluator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "cfg.DATASETS.TEST = (\"amodal_coco_val\",)\n",
    "cfg.OUTPUT_DIR = \"myAmodalCheckpoint\"\n",
    "cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, \"model_final.pth\")\n",
    "# cfg.MODEL.WEIGHTS =  model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")  # Let training initialize from model zoo\n",
    "cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.9   # set the testing threshold for this model\n",
    "predictor = DefaultPredictor(cfg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from detectron2.utils.visualizer import ColorMode\n",
    "import random\n",
    "from detectron2.data import DatasetCatalog\n",
    "from detectron2.utils.visualizer import Visualizer\n",
    "\n",
    "dataset_dicts = DatasetCatalog.get(\"amodal_coco_val\")\n",
    "for d in random.sample(dataset_dicts, 2):    \n",
    "    im = cv2.imread(d[\"file_name\"])\n",
    "    outputs = predictor(im)\n",
    "    v = Visualizer(im[:, :, ::-1],\n",
    "                   metadata=MetadataCatalog.get(\"amodal_coco_val\"), \n",
    "                   scale=0.8, \n",
    "                   instance_mode=ColorMode.IMAGE_BW   # remove the colors of unsegmented pixels\n",
    "    )\n",
    "    v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n",
    "    imshow(v.get_image()[:, :, ::-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
